#!/usr/bin/env python
# -*- coding: utf-8 -*-
#

import base64,re,time
from urlparse import urlparse,urlunparse

from pymongo.connection import Connection
from pymongo import ASCENDING,DESCENDING

from xml.sax import parseString
from xml.sax.handler import ContentHandler

from cStringIO import StringIO


def extractURL(xml_str) :
    class   URLHandler(ContentHandler) :
        def __init__(self) :
            self.url = None
            ## __init__()

        def startDocument(self) :
            self._path = []
            self._io = None
            ## startDocument()

        def endDocument(self) :
            pass
            ## endDocument()

        def startElement(self, name, attrs) :
            if self.url is not None : return
            self._path.append(name)
            if self.isUrlPath() :
                self._io = StringIO()
            ## startElement()

        def characters(self, content) :
            if self._io :
                self._io.write(content)
            ## characters()

        def endElement(self, name) :
            if self.url is not None : return
            if self._io :
                self.url = self._io.getvalue()
                self._io.close()
                self._io = None
            self._path.pop()
            ## endElement()


        def isUrlPath(self) :
            if len(self._path) != 3 : 
                return False
            if self._path == ['document', 'key', 'url' ] :
                return True
            return False
            ## isUrlPath()
        ## class Handler()

    handler = URLHandler()
    #parser = make_parser()
    #parser.setContentHandler( handler )
    #parser.parse( xml_str )
    parseString(xml_str, handler)
    
    url = handler.url

    return url
    ## extractURL()

class   Loader(object) :
    def __init__(self, config_dict) :
        self.config = config_dict
        host = self.config['host']
        port = int(self.config['port'])
        self.connection = Connection(host, port)

        dbname = self.config['db']
        self.db = self.connection[dbname]

        table_name = self.config['table']
        self.collection = self.db[ table_name ]

        self.progress = -1
        if 'progress' in self.config :
            self.progress = int(self.config['progress'])
        self.progress_count_down = self.progress
        self.progress_time_sum = 0
        self.progress_count_sum = 0
        ## __init__()

    def load(self, line) :
        if self.progress > 0 :
            time_start = time.time()

        try :
            key,data = self.parse(line)
            if not key : 
                return False
            self.collection.insert({'key':key,'data':data})
        except :
            return False

        if self.progress > 0 :
            self.progress_count_sum += 1
            self.progress_count_down -= 1
            self.progress_time_sum += time.time() - time_start
            if self.progress_count_down < 1 :
                self.showProgress()
                self.progress_count_down = self.progress

        return True
        ## load()

    def parse(self, data) :
        try :
            _date,_time,xml = data.split(' ',2)
        except :
            return None,None

        url = extractURL(xml)
        #print url

        key = self.getKeyFromUrl( url )
        data = base64.b64encode(xml)

        return key,data
        ## parse()

    def getKeyFromUrl(self, url) :
        url_reversed,scheme,port = reverseURL(url)

        return url_reversed
        ## getKeyFromUrl()

    def showProgress(self) :
        tps = 0
        if self.progress_time_sum > 0 :
            tps = self.progress_count_sum / self.progress_time_sum
        print '  - %9d records loaded in %7.2f seconds (TPS=%.2f) ...' % (self.progress_count_sum, self.progress_time_sum, tps)
        ## showProgress()

    ## class Loader

#######################################################################

IP_pattern = re.compile('([01]?\d\d?|2[0-4]\d|25[0-5])(\.([01]?\d\d?|2[0-4]\d|25[0-5])){3}', re.UNICODE)

def isIPAddress(hostname) :
    if None == IP_pattern.search(hostname) :
        return False
    else :
        return True
    ## isIPAddress()

def reverseHostname(hostname) :
    if not hostname :
        return hostname
    if isIPAddress(hostname) :
        return hostname

    f = hostname.split('.')
    f.reverse()
    return '.'.join(f)
    ## reverseHostname()

def reverseURL(url) :
    obj = urlparse(url)

    url_reversed = urlunparse( (obj.scheme, reverseHostname(obj.hostname), obj.path, obj.params, obj.query, obj.fragment) )
    return url_reversed[url_reversed.find('://') +3:], obj.scheme, obj.port
    ## reverseURL()


